package com.crawler.repitle;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SearchCrawler {
    /**
     * 正则表达式，取=和|之间的字符串，不包括=和|
     */
    private Pattern pattern = Pattern.compile("共(.*?)页，到第");

    public void crawl() {
        // 如果不设置将搜索环境变量
        String driverPath = App.class.getResource("/" ) + "exe/chromedriver.exe";
        driverPath = driverPath.substring(6);
        System.setProperty("webdriver.chrome.driver", driverPath);
        String url = "https://search.51job.com/list/020000,000000,0000,00,9,99,Spark,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";

        // InternetExplorerDriver()   浏览器
        // FirefoxDriver()            火狐浏览器
        //谷歌浏览器
        WebDriver driver = null;
        //创建chrome参数对象
        ChromeOptions options = new ChromeOptions();
        //浏览器后台运行
        options.addArguments("headless");
        driver = new ChromeDriver(options);
        driver.get(url);
        Document doc = Jsoup.parse(driver.getPageSource());
        Elements elements = doc.getElementsByClass("dw_page");
        Elements tds = elements.get(0).getElementsByClass("td");
        Element pageNumberElement = tds.get(0);
        Matcher m = pattern.matcher(pageNumberElement.ownText());
        if(m.find()) {
            int count = Integer.valueOf(m.group(1));
            for (int i = 1; i <= count; i++) {
                String subUrl = "https://search.51job.com/list/020000,000000,0000,00,9,99,Spark,2," + i + ".html?lang=c&amp;stype=1&amp;postchannel=0000&amp;workyear=99&amp;cotype=99&amp;degreefrom=99&amp;jobterm=99&amp;companysize=99&amp;lonlat=0%2C0&amp;radius=-1&amp;ord_field=1&amp;confirmdate=9&amp;fromType=&amp;dibiaoid=0&amp;address=&amp;line=&amp;specialarea=00&amp;from=&amp;welfare=";
                driver.get(subUrl);
                Document realDoc = Jsoup.parse(driver.getPageSource());
                Element dwTable = realDoc.getElementsByClass("dw_table").first();
                Elements elElements = dwTable.getElementsByClass("el");
                for (int j = 1; j < elElements.size(); j ++) {
                    Element element = elElements.get(j);
                    Element t1 = element.getElementsByClass("t1").first();
                    Element job = t1.getElementsByTag("a").first();
                    System.out.println(job.ownText());
                    System.out.println(job.attr("href"));

                    Element t2 = element.getElementsByClass("t2").first();
                    Element company = t2.getElementsByTag("a").first();
                    System.out.println(company.ownText());
                    System.out.println(company.attr("href"));

                    Element t3 = element.getElementsByClass("t3").first();
                    System.out.println(t3.ownText());

                    Element t4 = element.getElementsByClass("t4").first();
                    System.out.println(t4.ownText());

                    Element t5 = element.getElementsByClass("t5").first();
                    System.out.println(t5.ownText());

                }

            }

        }

//        int count = 0;
//        while (count < 100) {
//            try {
//
//                Element element = doc.getElementById("pushview");
//                Element liEle = element.getElementsByTag("li").get(1);
//                System.out.println(liEle);
//            } catch (Exception ex) {
//                System.err.println(ex.getMessage());
//            }
//            Thread.sleep(30000);
//            count++;
//        }

        //关闭浏览器
        driver.close();
        driver.quit();
    }

}
